package com.tledu;

import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.pipeline.JsonFilePipeline;
import us.codecraft.webmagic.processor.example.BaiduBaikePageProcessor;
import us.codecraft.webmagic.processor.example.GithubRepoPageProcessor;
import us.codecraft.webmagic.processor.example.ZhihuPageProcessor;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

public class TestWebMagic {
    public static void main(String[] args) {
        // 创建爬虫，创建的时候需要传递一个pageProcessor()
        Spider.create(new BaiduBaikePageProcessor())
                .addUrl("https://baike.baidu.com/item/%E4%BB%8A%E6%97%A5%E5%A4%B4%E6%9D%A1/4169373")
                // 这个数据会保存到哪里，jsonFile是webMagic帮我做的默认实现
                .addPipeline(new JsonFilePipeline("C:\\Users\\cyrus\\Desktop\\data"))
                .run();

    }
}
